{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4c6504b-2598-4894-a01f-50cd9074454f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Qwen2ForCausalLM(\n",
      "  (model): Qwen2Model(\n",
      "    (embed_tokens): Embedding(151936, 1536)\n",
      "    (layers): ModuleList(\n",
      "      (0-27): 28 x Qwen2DecoderLayer(\n",
      "        (self_attn): Qwen2SdpaAttention(\n",
      "          (q_proj): Linear(in_features=1536, out_features=1536, bias=True)\n",
      "          (k_proj): Linear(in_features=1536, out_features=256, bias=True)\n",
      "          (v_proj): Linear(in_features=1536, out_features=256, bias=True)\n",
      "          (o_proj): Linear(in_features=1536, out_features=1536, bias=False)\n",
      "          (rotary_emb): Qwen2RotaryEmbedding()\n",
      "        )\n",
      "        (mlp): Qwen2MLP(\n",
      "          (gate_proj): Linear(in_features=1536, out_features=8960, bias=False)\n",
      "          (up_proj): Linear(in_features=1536, out_features=8960, bias=False)\n",
      "          (down_proj): Linear(in_features=8960, out_features=1536, bias=False)\n",
      "          (act_fn): SiLU()\n",
      "        )\n",
      "        (input_layernorm): Qwen2RMSNorm()\n",
      "        (post_attention_layernorm): Qwen2RMSNorm()\n",
      "      )\n",
      "    )\n",
      "    (norm): Qwen2RMSNorm()\n",
      "  )\n",
      "  (lm_head): Linear(in_features=1536, out_features=151936, bias=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Qwen2TokenizerFast(name_or_path='D:\\ModelSpace\\Qwen2.5\\Qwen2.5-1.5B-Instruct', vocab_size=151643, model_max_length=131072, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={\n",
      "\t151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151646: AddedToken(\"<|object_ref_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151647: AddedToken(\"<|object_ref_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151648: AddedToken(\"<|box_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151649: AddedToken(\"<|box_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151650: AddedToken(\"<|quad_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151651: AddedToken(\"<|quad_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151652: AddedToken(\"<|vision_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151653: AddedToken(\"<|vision_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151654: AddedToken(\"<|vision_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151655: AddedToken(\"<|image_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151656: AddedToken(\"<|video_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t151657: AddedToken(\"<tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151658: AddedToken(\"</tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151659: AddedToken(\"<|fim_prefix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151660: AddedToken(\"<|fim_middle|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151661: AddedToken(\"<|fim_suffix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151662: AddedToken(\"<|fim_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151663: AddedToken(\"<|repo_name|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "\t151664: AddedToken(\"<|file_sep|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoModelForCausalLM\n",
    "\n",
    "# 模型文件目录\n",
    "model_dir = os.path.join('D:', os.path.sep, 'ModelSpace', 'Qwen2.5', 'Qwen2.5-1.5B-Instruct')\n",
    "\n",
    "# 初始化模型\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_dir,\n",
    "    torch_dtype=\"auto\",\n",
    "    device_map=\"auto\",\n",
    "    local_files_only=True,\n",
    ")\n",
    "print(model)\n",
    "\n",
    "# 初始化分词器\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    model_dir,\n",
    "    local_files_only=True,\n",
    ")\n",
    "print(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "80436ef6-ee02-46bc-82be-ca8415002255",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Transform', 'ers', 'åĪĨ', 'è¯į', 'ï¼ļ', 'åı°é£İ', 'åıĪ', 'åıĮ', 'åı', 'Ĵ', 'åıķ', 'æĿ¥äºĨ', 'ï¼ģ']\n"
     ]
    }
   ],
   "source": [
    "text = 'Transformers分词：台风又双叒叕来了！'\n",
    "tokens = tokenizer.tokenize(text)\n",
    "\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c72b110e-f161-4c31-929d-0c1171684d60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8963, 388, 17177, 99689, 5122, 108118, 99518, 99493, 5758, 240, 122378, 101161, 6313]\n"
     ]
    }
   ],
   "source": [
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7557708a-2b14-448f-84cc-6c74957b4166",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8963, 388, 17177, 99689, 5122, 108118, 99518, 99493, 5758, 240, 122378, 101161, 6313]\n"
     ]
    }
   ],
   "source": [
    "token_ids = tokenizer.encode(text)\n",
    "\n",
    "print(token_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "52df66b3-d415-4034-b016-09f4f3e7a95a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformers分词：台风又双叒叕来了！\n"
     ]
    }
   ],
   "source": [
    "token_text = tokenizer.decode(token_ids)\n",
    "\n",
    "print(token_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3f93e552-146c-4782-b3ad-263307f45198",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "新增加 2个普通Token到词表。\n"
     ]
    }
   ],
   "source": [
    "# 添加普通Token：如果词表存在则忽略\n",
    "new_tokens = [\"老牛同学\", \"imxulin\"]\n",
    "new_tokens = set(new_tokens) - set(tokenizer.vocab.keys())\n",
    "\n",
    "num_add_tokens = tokenizer.add_tokens(list(new_tokens))\n",
    "\n",
    "print(f'新增加 {num_add_tokens}个普通Token到词表。')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b9b93348-5631-41a1-ba50-dd0935fd8f32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "新增加 1个特殊Token到词表。\n",
      "特殊Token值：[LNTX]\n"
     ]
    }
   ],
   "source": [
    "mew_special_tokens = {'cls_token': '[LNTX]'}\n",
    "\n",
    "num_add_spec_tokens = tokenizer.add_special_tokens(mew_special_tokens)\n",
    "\n",
    "print(f'新增加 {num_add_spec_tokens}个特殊Token到词表。')\n",
    "print(f'特殊Token值：{tokenizer.cls_token}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "73ff8e4d-780d-44bd-b4e4-50890becad5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['å¤§å®¶', '[LNTX]', 'å¥½', 'ï¼Į', 'æĪĳæĺ¯', '老牛同学', 'ï¼Į', 'ä»ĸ', 'æĺ¯ä¸Ģä½į', 'å¤§', 'æ¨¡åŀĭ', '[LNTX]', 'çĪ±å¥½èĢħ', 'ï¼ģ']\n"
     ]
    }
   ],
   "source": [
    "text = '大家[LNTX]好，我是老牛同学，他是一位大模型[LNTX]爱好者！'\n",
    "tokens = tokenizer.tokenize(text)\n",
    "\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ed97bfab-1646-43ee-92d8-cbed70da716c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "调整前：torch.Size([151936, 1536])\n",
      "调整后：torch.Size([151668, 1536])\n"
     ]
    }
   ],
   "source": [
    "print(f'调整前：{model.model.embed_tokens.weight.size()}')\n",
    "\n",
    "model.resize_token_embeddings(len(tokenizer))\n",
    "\n",
    "print(f'调整后：{model.model.embed_tokens.weight.size()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cde065d-7909-4948-a042-48bc133a832a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
